package com.nulldev.util.web.HttpServer3.extensions.crawlerBlock;

import com.nulldev.util.VariableAPI.ArrayUtils;
import com.nulldev.util.data.Variables;

public class CrawlerBlock {

	private static final String[] CRAWLER_USER_AGENTS =
		{ "Googlebot-Image/1.0", "Googlebot/2.1", "+http://www.google.com/bot.html", "Googlebot-Mobile/2.1", "Googlebot-News", "Googlebot-Video/1.0",

				"AdsBot-Google (+http://www.google.com/adsbot.html)", "AdsBot-Google-Mobile-Apps", "AdsBot-Google-Mobile",
				"+http://www.google.com/mobile/adsbot.html",

				"Feedfetcher-Google",

				"Mediapartners-Google",

				"APIs-Google", "https://developers.google.com/webmasters/APIs-Google.html",

				"adidxbot/2.0", "http://www.bing.com/bingbot.htm", "bingbot/2.0",

				"Slurp/3.0", "Slurp", "Slurp China",

				"WGETbot/1.0", "LinkedInBot/1.0", "phpcrawl", "adidxbot/1.1", "msnbot", "FAST-WebCrawler", "FAST Enterprise Crawler", "Ask Jeeves/Teoma",
				"ConveraCrawler", "Seekbot", "Gigabot", "GigablastOpenSource", "Alexabot", "Exabot", "ia_archiver", "HTTrack", "grub-client", "Speedy Spider",
				"findlinks", "yacybot", "MJ12bot",

				"YandexBot", "YandexImages", "YandexMobileBot", "YandexMetrika", "YandexTurbo", "YandexImageResizer", "YandexVideo", "YandexAdNet",
				"YandexBlogs", "YandexCalendar", "YandexDirect", "YaDirectFetcher", "YandexForDomain", "YandexMarket", "YandexMedia",
				"YandexMobileScreenShotBot", "YandexNews", "YandexOntoDB", "YandexPagechecker", "YandexPartner", "YandexRCA", "YandexSearchShop",
				"YandexSitelinks", "YandexSpravBot", "YandexTracker", "YandexVertis", "YandexVerticals", "YandexWebmaster", "YandexScreenshotBot",

				"facebookexternalhit", "DuckDuckBot", "360Spider", "Twitterbot", "Qwantify", "archive.org_bot", "Applebot", "Y!J", "Cliqzbot", "MojeekBot",
				"Bytespider", "Amazonbot",

				"Mastodon/", };

	public static boolean isCrawler(final String userAgent) {
		Variables.requireNonNull(userAgent, "userAgent");
		return ArrayUtils.StringArrayUtils.containsInString(CRAWLER_USER_AGENTS, userAgent);
	}
}
